package com.zxkj.service;

import com.zxkj.domain.dao.SciDeptDao;
import com.zxkj.domain.entity.*;
import com.zxkj.domain.entity.PublicationImpactfactor;
import com.zxkj.domain.entity.PublicationSubjectarea;
import com.zxkj.domain.entity.SciCorp;
import com.zxkj.domain.entity.SciDoc;
import com.zxkj.domain.mapper.*;
import com.zxkj.domain2.entity.SciDocAdd;
import com.zxkj.domain2.mapper.SciDocAddRepository;
import com.zxkj.exception.ListCountTooLongException;
import com.zxkj.solr.model.SciDocSolrModel;
import com.zxkj.solr.repository.SciDocSolrRepository;
import com.zxkj.utils.CommonUtils;
import com.zxkj.utils.ReadHtmlHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.core.query.SimpleQuery;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by Administrator on 2017/8/4.
 */
@Service
public class FixServiceImpl implements FixService {


    private static final Logger logger = LoggerFactory.getLogger(FixServiceImpl.class.getName());

    @Autowired
    private SciAuRepository sciAuRepository;
    @Autowired
    private SciAdRepository sciAdRepository;
    @Autowired
    private SciDocRepository sciDocRepository;
    @Autowired
    private SciDocAddRepository sciDocAddRepository;
    @Autowired
    private SciMapAdCorporationRepository sciMapAdCorporationRepository;
    @Autowired
    private SciMapAuadRepository sciMapAuadRepository;
    @Autowired
    private SciDocCopyRepository sciDocCopyRepository;
    @Autowired
    private PublicationService publicationService;
    @Autowired
    private SolrTemplate solrTemplate;
    @Autowired
    private SolrTemplate solrTemplate2;
    @Autowired
    private FixUnsaveUtRepository fixUnsaveUtRepository;
    @Autowired
    private SciUpdateLogRepository sciUpdateLogRepository;
    @Autowired
    private CorporationTermRepository corporationTermRepository;
    @Autowired
    private CorporationRepository corporationRepository;
    @Autowired
    private CorporationDeptRepository corporationDeptRepository;
    @Autowired
    private CorporationDeptTermRepository corporationDeptTermRepository;
    @Autowired
    private SciMapDeptDocRepository sciMapDeptDocRepository;
    @Autowired
    private SciMapDeptAdRepository sciMapDeptAdRepository;
    @Autowired
    private CorporationAuRepository corporationAuRepository;
    @Autowired
    private CorporationMapDeptAuRepository corporationMapDeptAuRepository;
    @Autowired
    private SciMapAuCorpAuRepository sciMapAuCorpAuRepository;
    @Autowired
    private ShowCorpRepository showCorpRepository;
    @Autowired
    private SciDeptDao sciDeptDao;

    private HashMap<String, List<String>> publication_subjectarea_temp;//publication_subjectarea表数据
    private List<PublicationImpactfactor> publication_impactfactor_temp;//publication_impactfactor表数据
    private List<DocumentAddress> documentAddresses_by_ut_temp = new ArrayList<>();//DocumentAddress表数据(按ut取出的数据)
    //待插入sciMapAuads表缓存
    private List<SciMapAuAd> sciMapAuAds_temp = new ArrayList<>();
    //待插入SciAu表缓存
    private List<SciAu> sciAus_temp = new ArrayList<>();
    //待插入SciAd表缓存
    private List<SciAd> sciAds_temp = new ArrayList<>();
    //待插入SciMapAdCorporation表缓存
    private List<SciMapAdCorporation> sciMapAdCorps_temp = new ArrayList<>();
    //待插入SciMapuUtDoi表缓存
    private List<SciMapUtDoi> sciMapUtDoi_temp = new ArrayList<>();
    private List<SciMapDeptDoc> sciMapdeptDoc_temp = new ArrayList<>();
    private List<SciMapDeptAd> sciMapDeptAd_temp = new ArrayList<>();
    private List<SciDoc> sciList = new ArrayList<>();
    private List<SciDoc> sciList_update = new ArrayList<>();
    private List<SciDoc> sciList_no_exist = new ArrayList<>();
    private List<SciDocSolrModel> sciDocSolrModelList = new ArrayList<>();
    private List<SciAd> sciAds_solr = new ArrayList<>();
    private List<SciAu> sciAus_solr = new ArrayList<>();
    //    private List<SciMapAdCorp> sciMapAdCorps_solr = new ArrayList<>();
    private List<String> reprintAuthor_temp = new ArrayList<>();//通讯作者临时变量
    private List<FixUnsaveUt> fixUnsaveUts_wait_delete = new ArrayList<>();//已插入ut，待删除的fix_unsave_ut
    private List<SciDocAdd> sciDocAdd_wait_delete = new ArrayList<>();//待删除的sci_doc_add
    private List<CorporationAu> corporationAus_temp = new ArrayList<>();
    private List<CorporationMapDeptAu> corporationMapDeptAu_temp = new ArrayList<>();
    private List<SciMapAuCorpAu> sciMapAuCorpAus_temp = new ArrayList<>();

    private Map<Integer, Corporation> corporationMap = new HashMap<>();
    private Map<String, List<CorporationTerm>> corporationTermMap = new HashMap<>();
    private Map<Integer, Map<String, List<CorporationDept>>> corporationDeptTermMap = new HashMap<>();//key: corpId; value: term和deptId的map
    private Map<String, String> corpFullNameAndIdMap = new HashMap<>();

    private static Integer listSizeMax = 1000;
    @Value("${author-size-max}")
    private Integer authorSizeMax;//当作者大于这个值的时候抛异常
    private int updateCnt = 0;//更新条数

    @Value("${my.page-size}")
    private Integer pageSize;
    @Value("${corpId}")
    private Integer corpId;
    @Value("${need.insert.mysql}")
    private boolean needInsertMysql;
    @Value("${need.insert.solr}")
    private boolean needInsertSolr;
    @Value("${need.updateNewUt}")
    private byte needUpdateNewUt;
    @Value("${solr.core.name}")
    private String solrCoreName;
    @Value("${solr.core.name2}")
    private String solrCoreName2;
    @Value("${is.print.line}")
    private boolean isPrintLine;
    @Value("${docid.list}")
    private String docIdList;
    @Value("${spring.data.solr.host2_query}")
    private String host2_query;

    /**
     * 更新不存在的ut的数据之后，返回所有的scidoc
     *
     * @param nowpage
     * @return
     */
    private void updateNewUt(Integer nowpage, Integer endPage) {
        sciList_update.clear();
        sciList_no_exist.clear();
        if (nowpage.compareTo(endPage) > 0 && endPage.compareTo(0) > 0) {
            return;
        }
        List<SciDocCopy> list_copy;
//        PageRequest pageRequest = new PageRequest(nowpage, pageSize, new Sort(Sort.Direction.ASC, "docId"));
        int startNum = nowpage * pageSize + 1;
        int endNum = (nowpage + 1) * pageSize;
        list_copy = sciDocCopyRepository.findAll(startNum, endNum);
        SciDoc maxNum = sciDocRepository.findMaxNum();
        int num = maxNum == null ? 0 : maxNum.getNum();
        for (SciDocCopy sciDocCopy : list_copy) {
            SciDoc sciDoc = sciDocRepository.findByUT(sciDocCopy.getUt());
//            if (sciDoc == null || sciDoc.getDocId() == null || sciDoc.getDocId().isEmpty()) {
//                //如果ut号不存在，则插入新的数据
//                sciDoc = new SciDoc();
//                BeanUtils.copyProperties(sciDocCopy, sciDoc);
//                sciDoc.setNum(++num);
//                sciList_no_exist.add(sciDoc);
//            } else {
//                //ut号存在则更新时间和被引次数等数据
//                if (sciDoc.getCitedcnt().intValue() != sciDocCopy.getCitedcnt().intValue()) {
//                    sciDoc.setCreateon(new Date());
//                    sciDoc.setHtml(sciDocCopy.getHtml());
//                    sciDoc.setCitedcnt(sciDocCopy.getCitedcnt());
//                    sciList_update.add(sciDoc);
//                }
//            }
            sciList.add(sciDoc);
        }
        if (sciList_no_exist.size() > 0) {
            logger.warn("不存在的ut个数:" + sciList_no_exist.size());
            sciDocRepository.save(sciList_no_exist);
        }
        if (sciList_update.size() > 0) {
            logger.warn("待更新的doc个数:" + sciList_update.size());
            sciDocRepository.save(sciList_update);
        }
    }

    /**
     * 获取全部的sci_doc表数据
     *
     * @param nowpage
     * @param endPage
     */
    private void findAllSciDoc(Integer nowpage, Integer endPage) {
        if (nowpage.compareTo(endPage) > 0 && endPage.compareTo(0) > 0) {
            return;
        }
        int startNum = nowpage * pageSize + 1;
        int endNum = (nowpage + 1) * pageSize;
        sciList = sciDocRepository.findAll(startNum, endNum);
    }

    /**
     * 获取特定的solr数据(从solr的第二个数据源中)
     *
     * @param nowpage
     * @param endPage
     */
    private void findPartFromSolr(Integer nowpage, Integer endPage) {
        if (nowpage.compareTo(endPage) > 0 && endPage.compareTo(0) > 0) {
            return;
        }
        SimpleQuery sq = new SimpleQuery(host2_query, new PageRequest(nowpage, pageSize));

        Page<SciDocSolrModel> result = solrTemplate2.queryForPage(solrCoreName2, sq, SciDocSolrModel.class);
        if (result != null && result.getContent().size() > 0) {
            for (SciDocSolrModel sciDocSolrModel1 : result.getContent()) {
                SciDoc sciDoc = sciDocRepository.findByUT(sciDocSolrModel1.getUt());
                sciList.add(sciDoc);
            }
        }
    }

    /**
     * 根据fix_unsave_ut获取sci_doc的表数据
     *
     * @param nowpage
     * @param endPage
     */
    private void findAllSciDocFixUnSaveUt(Integer nowpage, Integer endPage) {
        fixUnsaveUts_wait_delete.clear();
        if (nowpage.compareTo(endPage) > 0 && endPage.compareTo(0) > 0) {
            return;
        }
        Page<FixUnsaveUt> fixUnsaveUts = fixUnsaveUtRepository.findAll(new PageRequest(0, pageSize));
        for (FixUnsaveUt fixUnsaveUt : fixUnsaveUts.getContent()) {
            SciDoc sciDoc = sciDocRepository.findByUT(fixUnsaveUt.getUt());
            sciList.add(sciDoc);
        }
    }

    /**
     * 根据fix_unsave_ut获取sci_doc_copy的表数据
     *
     * @param nowpage
     * @param endPage
     */
    private void findAllSciDocCopyFixUnSaveUt(Integer nowpage, Integer endPage) {
        fixUnsaveUts_wait_delete.clear();
        sciList_update.clear();
        sciList_no_exist.clear();
        if (nowpage.compareTo(endPage) > 0 && endPage.compareTo(0) > 0) {
            return;
        }
        int startNum = nowpage * pageSize + 1;
        int endNum = (nowpage + 1) * pageSize;
        List<FixUnsaveUt> fixUnsaveUts = fixUnsaveUtRepository.findAll();
        for (FixUnsaveUt fixUnsaveUt : fixUnsaveUts) {
            SciDocCopy sciDocCopy = sciDocCopyRepository.findByUT(fixUnsaveUt.getUt());
            SciDoc maxNum = sciDocRepository.findMaxNum();
            int num = maxNum == null ? 0 : maxNum.getNum();
            SciDoc sciDoc = sciDocRepository.findByUT(sciDocCopy.getUt());
            if (sciDoc == null || sciDoc.getDocId() == null || sciDoc.getDocId().isEmpty()) {
                //如果ut号不存在，则插入新的数据
                sciDoc = new SciDoc();
                BeanUtils.copyProperties(sciDocCopy, sciDoc);
                sciDoc.setNum(++num);
                sciList_no_exist.add(sciDoc);
            } else {
                //ut号存在则更新时间和被引次数等数据
                if (sciDoc.getCitedcnt().intValue() != sciDocCopy.getCitedcnt().intValue()) {
                    sciDoc.setCreateon(new Date());
                    sciDoc.setHtml(sciDocCopy.getHtml());
                    sciDoc.setCitedcnt(sciDocCopy.getCitedcnt());
                    sciList_update.add(sciDoc);
                }
                sciList.add(sciDoc);
                sciDoc = null;
            }
        }
        if (sciList_no_exist.size() > 0) {
            logger.warn("不存在的ut个数:" + sciList_no_exist.size());
            sciDocRepository.save(sciList_no_exist);
        }
        if (sciList_update.size() > 0) {
            logger.warn("待更新的doc个数:" + sciList_update.size());
            sciDocRepository.save(sciList_update);
        }
    }

    /**
     * 获取配置文件中的docId数据
     */
    private void findDocInProperties() {
        String[] docIdArr = docIdList.split(",");
        if (docIdArr != null) {
            for (String docId : docIdArr) {
                SciDoc sciDoc = sciDocRepository.findOne(docId);
                if (sciDoc != null && sciDoc.getDocId() != null) {
                    sciList.add(sciDoc);
                    sciDoc = null;
                }
            }
        }
    }

    /**
     * 获取sci_doc_add的数据更新到服务器中去
     */
    private void updateDocAddToServer() {
        sciList_update.clear();
        sciList_no_exist.clear();
        sciDocAdd_wait_delete.clear();
        List<SciDocAdd> docAdds = sciDocAddRepository.getDocList(pageSize);
        if (docAdds != null && docAdds.size() > 0) {
            SciDoc maxNum = sciDocRepository.findMaxNum();
            int num = maxNum == null ? 0 : maxNum.getNum();
            Set<String> ut_set = new HashSet<>();//ut的set，防止sci_doc_add中有重复的ut
            for (SciDocAdd sciDocAdd : docAdds) {
                if (!ut_set.contains(sciDocAdd.getUt())) {
                    ut_set.add(sciDocAdd.getUt());
                    SciDoc sciDoc = sciDocRepository.findByUT(sciDocAdd.getUt());
                    if (sciDoc == null || sciDoc.getDocId() == null || sciDoc.getDocId().isEmpty()) {
                        //如果ut号不存在，则插入新的数据
                        sciDoc = new SciDoc();
                        BeanUtils.copyProperties(sciDocAdd, sciDoc);
                        sciDoc.setNum(++num);
                        sciList.add(sciDoc);
                        sciList_no_exist.add(sciDoc);
                    } else {
                        //ut号存在则更新时间和被引次数等数据
                        if (sciDoc.getCitedcnt().intValue() < sciDocAdd.getCitedcnt().intValue()) {
                            sciDoc.setCitedon(new Date());
                            sciDoc.setHtml(sciDocAdd.getHtml());
                            sciDoc.setCitedcnt(sciDocAdd.getCitedcnt());
                            sciList_update.add(sciDoc);
                            sciList.add(sciDoc);
                        }
                    }
                }
                sciDocAdd_wait_delete.add(sciDocAdd);
            }
            if (sciList_no_exist.size() > 0) {
                logger.warn("不存在的ut个数:" + sciList_no_exist.size());
                sciDocRepository.save(sciList_no_exist);
            }
            if (sciList_update.size() > 0) {
                logger.warn("待更新的doc个数:" + sciList_update.size());
                sciDocRepository.save(sciList_update);
            }
        }

    }

    //获取term表内容，放入内存中
    private void getTermTable() {
        /*logger.warn("取出sci_corp_term表数据，放入内存中......");
        List<SciCorpTerm> term_list = sciCorpTermRepository.findAll();
        termTable = new HashMap<>();
        for (SciCorpTerm sciCorpTerm : term_list) {
            String termbase = CommonUtils.trimAndRegex2(sciCorpTerm.getTermbase());
            if (termTable.containsKey(termbase)) {
                termTable.get(termbase).add(sciCorpTerm.getCorpId());
            } else {
                List<Integer> corpid_list = new ArrayList<>();
                corpid_list.add(sciCorpTerm.getCorpId());
                termTable.put(termbase, corpid_list);
            }
        }
        logger.warn("取出sci_corp表数据，放入内存中......");
        List<SciCorp> list = sciCorpRepository.findAll();
        sciCorps_temp = new HashMap<>();
        sciCorps_temp2 = new HashMap<>();
        corp_chinese_temp = new HashMap<>();
        for (SciCorp sciCorp : list) {
            sciCorps_temp.put(sciCorp.getCorpId(), sciCorp);
            sciCorps_temp2.put(CommonUtils.trimAndRegex2(sciCorp.getName()), sciCorp.getCorpId());
            //将中国的中英文对照存入内存中
            if (!StringUtils.isEmpty(sciCorp.getChinese())) {
                corp_chinese_temp.put(sciCorp.getName(), sciCorp.getChinese());
            }
        }
        logger.warn("取出sci_corp表完毕");*/

        logger.warn("取出publication_impactfactor表数据，放入内存中......");
        publication_impactfactor_temp = publicationService.getPublicationImpactfactor();
        logger.warn("取出publication_impactfactor表完毕");

        logger.warn("取出publication_subjectarea表数据，放入内存中......");
        List<PublicationSubjectarea> publicationSubjectareas = publicationService.getPublicationSubjectarea();
        publication_subjectarea_temp = new HashMap<>();
        for (PublicationSubjectarea item : publicationSubjectareas) {
            String item_key = item.getISSN() + item.getCategory();
            if (publication_subjectarea_temp.containsKey(item_key)) {
                publication_subjectarea_temp.get(item_key).add(item.getName());
            } else {
                List<String> item_value = new ArrayList<>();
                item_value.add(item.getName());
                publication_subjectarea_temp.put(item_key, item_value);
            }
        }
        logger.warn("取出publication_subjectarea表完毕");

        /*logger.warn("取出addr表数据，放入内存中......");
        List<Addr> addrs = addrService.getAllAddrGtZero();
        addr_temp = new HashMap<>();
        for (Addr item : addrs) {
            if (item.getCorpId() != null && item.getCorpId().intValue() > 0) {
                String addr_trim = CommonUtils.trimAndRegex2(item.getAddress());
                if (addr_temp.containsKey(addr_trim)) {
                    addr_temp.get(addr_trim).add(item.getCorpId());
                } else {
                    List<Integer> corpIdList = new ArrayList<>();
                    corpIdList.add(item.getCorpId());
                    addr_temp.put(addr_trim, corpIdList);
                }
            }
        }
        logger.warn("取出addr表完毕");*/

        logger.warn("取出corpration_term表数据，放入内存中......");
        List<CorporationTerm> corporationTermList = corporationTermRepository.findAll();
        for (CorporationTerm corporationTerm : corporationTermList) {
            String[] termBase_arr = CommonUtils.trimAndRegex2(corporationTerm.getTermBase()).split("\\$");
            if (!StringUtils.isEmpty(corporationTerm.getTermAnd())) {
                corporationTerm.setTermAnd(".*(" + CommonUtils.trimAndRegex3(corporationTerm.getTermAnd()) + ").*");
            }
            if (!StringUtils.isEmpty(corporationTerm.getTermNot())) {
                corporationTerm.setTermNot(".*(" + CommonUtils.trimAndRegex3(corporationTerm.getTermNot()) + ").*");
            }
            for (String termBase : termBase_arr) {
                termBase = termBase.trim();
                termBase = StringUtils.trimTrailingCharacter(termBase, ',');
                termBase = StringUtils.trimLeadingCharacter(termBase, ',');
                if (corporationTermMap.containsKey(termBase)) {
                    corporationTermMap.get(termBase).add(corporationTerm);
                } else {
                    List<CorporationTerm> corporationTerms = new ArrayList<>();
                    corporationTerms.add(corporationTerm);
                    corporationTermMap.put(termBase, corporationTerms);
                }
            }
        }
        logger.warn("取出corpration_term表完毕(" + corporationTermMap.size() + ")");

        logger.warn("取出corpration表数据，放入内存中......");
        List<Corporation> corporationList = corporationRepository.findAll();
        for (Corporation corporation : corporationList) {
            corporationMap.put(corporation.getId(), corporation);
        }
        logger.warn("取出corpration表完毕(" + corporationList.size() + ")");

        logger.warn("取出corpration_dept表数据，放入内存中......");
        Map<Integer, Set<CorporationDept>> corporationDeptMap = new HashMap<>();
        List<CorporationDept> corporationDeptList = corporationDeptRepository.findAll();
        for (CorporationDept corporationDept : corporationDeptList) {
            Integer corpId = corporationDept.getCorpId();
            if (corporationDeptMap.containsKey(corpId)) {
                corporationDeptMap.get(corpId).add(corporationDept);
            } else {
                HashSet<CorporationDept> dept_set = new HashSet<>();
                dept_set.add(corporationDept);
                corporationDeptMap.put(corpId, dept_set);
            }
        }
        logger.warn("取出corpration_dept表完毕(" + corporationDeptList.size() + ")");

        logger.warn("取出corporation_dept_term表数据，放入内存中......");
        List<CorporationDeptTerm> corporationDeptTermList = corporationDeptTermRepository.findAll();
        Map<Integer, HashSet<String>> dept_term_base_map = new HashMap<>();//部门id，term字符串
        for (CorporationDeptTerm corporationDeptTerm : corporationDeptTermList) {
            Integer deptId = corporationDeptTerm.getDeptId();
            String term_base_trim = CommonUtils.trimAndRegex2(corporationDeptTerm.getTermBase()).trim();
            if (dept_term_base_map.containsKey(deptId)) {
                dept_term_base_map.get(deptId).add(term_base_trim);
            } else {
                HashSet<String> term_set = new HashSet<>();
                term_set.add(term_base_trim);
                dept_term_base_map.put(deptId, term_set);
            }
        }
        Map<String, List<CorporationDept>> term_deptId_map;
        for (Integer corpId : corporationDeptMap.keySet()) {
            term_deptId_map = new HashMap<>();
            Set<CorporationDept> dept_set = corporationDeptMap.get(corpId);
            for (CorporationDept dept : dept_set) {
                Integer deptId = dept.getId();
                if (dept_term_base_map.containsKey(deptId)) {
                    HashSet<String> term_set = dept_term_base_map.get(deptId);
                    for (String term_base : term_set) {
                        if (term_deptId_map.containsKey(term_base)) {
                            term_deptId_map.get(term_base).add(dept);
                        } else {
                            List<CorporationDept> deptIdList = new ArrayList<>();
                            deptIdList.add(dept);
                            term_deptId_map.put(term_base, deptIdList);
                        }
                    }
                }
            }
            corporationDeptTermMap.put(corpId, term_deptId_map);
        }
        logger.warn("取出corporation_dept_term表完毕");


    }

    /**
     * 根据ut提取所有html，插入adad表
     */
    @Override
    public void fixMain(Integer startPage, Integer endPage) {
        //获取term表内容，放入内存中
        getTermTable();
        Integer page = startPage;
        while (true) {
            sciMapAuAds_temp.clear();
            sciAus_temp.clear();
            sciAds_temp.clear();
            sciMapAdCorps_temp.clear();
            sciMapUtDoi_temp.clear();
            sciDocSolrModelList.clear();
            sciList.clear();
            sciMapdeptDoc_temp.clear();
            switch (needUpdateNewUt) {
                case 0:
                    updateNewUt(page, endPage);//获取需要修复的doc实体数据
                    break;
                case 1:
                    findAllSciDoc(page, endPage);
                    break;
                case 2:
                    findPartFromSolr(page, endPage);
                    break;
                case 3:
                    findAllSciDocFixUnSaveUt(page, endPage);
                    break;
                case 4:
                    findAllSciDocCopyFixUnSaveUt(page, endPage);
                    break;
                case 5:
                    findDocInProperties();
                    break;
                case 6:
                    updateDocAddToServer();
                    break;
            }
            if (sciList != null && sciList.size() > 0) {
                updateCnt += sciList.size();
                logger.warn("当前第" + (page + 1) + "页");
                logger.warn("开始处理auad数据......");
                for (SciDoc sciDoc : sciList) {
//                logger.warn("####UT：" + sciDoc.getUt() + "####");
                    try {
                        InsertSciAuadFromHtml(sciDoc);
                        if (isPrintLine) {
                            System.out.print("|");
                        }
                        if (needUpdateNewUt == 3 || needUpdateNewUt == 4) {
                            FixUnsaveUt fixUnsaveUt = new FixUnsaveUt();
                            fixUnsaveUt.setUt(sciDoc.getUt());
                            fixUnsaveUts_wait_delete.add(fixUnsaveUt);
                        }
                    } catch (ListCountTooLongException e) {
                        FixUnsaveUt fixUnsaveUt = new FixUnsaveUt();
                        fixUnsaveUt.setUt(e.getUt());
                        fixUnsaveUt.setCnt(e.getListCount());
                        fixUnsaveUtRepository.save(fixUnsaveUt);
                    }
                }
                if (needInsertMysql) {
                    //批量插入表数据
                    logger.warn("开始批量插入sci_ad表，共" + sciAds_temp.size() + "条数据");
                    if (sciAds_temp.size() > 0) {
                        commitSciAd();
                    }
                    logger.warn("开始批量插入sci_map_ad_corp表，共" + sciMapAdCorps_temp.size() + "条数据");
                    if (sciMapAdCorps_temp.size() > 0) {
                        commitSciMapAdCorp();
                    }
                    logger.warn("开始批量插入sci_map_dept_ad表，共" + sciMapDeptAd_temp.size() + "条数据");
                    if (sciMapDeptAd_temp.size() > 0) {
                        commitSciMapDeptAd();
                    }
                    logger.warn("开始批量插入sci_au表，共" + sciAus_temp.size() + "条数据");
                    if (sciAus_temp.size() > 0) {
                        commitSciAu();
                    }
                    logger.warn("开始批量插入sci_map_au_ad表，共" + sciMapAuAds_temp.size() + "条数据");
                    if (sciMapAuAds_temp.size() > 0) {
                        commitSciMapAuAd();
                    }
                    logger.warn("开始批量插入sci_map_dept_doc表，共" + sciMapdeptDoc_temp.size() + "条数据");
                    if (sciMapdeptDoc_temp.size() > 0) {
                        commitSciMapDeptDoc();
                    }

                /*logger.warn("开始批量插入sci_map_ut_doi表，共" + sciMapUtDoi_temp.size() + "条数据");
                if (sciMapUtDoi_temp.size() > 0) {
                    commitSciMapUtDoi();
                }*/
                }
                if (needInsertSolr) {
                    logger.warn("开始插入solr数据......(" + sciDocSolrModelList.size() + "条)");
//                    sciDocSolrRepository.save(sciDocSolrModelList);
                    solrTemplate.saveBeans(solrCoreName, sciDocSolrModelList);
                    solrTemplate.commit(solrCoreName);
                    logger.warn("插入solr完成");
                }
                if (fixUnsaveUts_wait_delete.size() > 0) {
                    logger.warn("开始清除fix_unsave_ut数据......(" + fixUnsaveUts_wait_delete.size() + "条)");
                    fixUnsaveUtRepository.delete(fixUnsaveUts_wait_delete);
                    logger.warn("清除完成");
                }
            }
            if (sciDocAdd_wait_delete.size() > 0) {
                logger.warn("开始清除sci_doc_add数据......(" + sciDocAdd_wait_delete.size() + "条)");
                sciDocAddRepository.delete(sciDocAdd_wait_delete);
                logger.warn("清除完成");
            } else {
                if (needUpdateNewUt == 6) {
                    logger.warn("最后一页已完成");
                    break;
                }
            }
            if (needUpdateNewUt != 6) {
                if ((endPage.intValue() == 0 && (sciList == null || sciList.size() == 0)) || (endPage.intValue() > 0 && (page.intValue() == endPage.intValue()))) {//当前为最后一页退出
                    logger.warn("最后一页已完成");
                    break;
                }
            }
            page++;
            if (needUpdateNewUt == 5) {
                break;
            }
        }
        if (needUpdateNewUt == 6) {
            logger.warn("开始添加更新记录sci_update_log表");
            SciUpdateLog sciUpdateLog = new SciUpdateLog();
            sciUpdateLog.setUpdateTime(new Date());
            sciUpdateLog.setUpdateCnt(updateCnt);
            sciUpdateLogRepository.save(sciUpdateLog);
            logger.warn("开始将学校的待更新缓存字段置为1");
            showCorpRepository.updateNeedUpdateToOne();
        }
    }


    /**
     * 根据html插入auad表
     */
    private void InsertSciAuadFromHtml(SciDoc sciDoc) throws ListCountTooLongException {
        sciAds_solr.clear();
        sciAus_solr.clear();
        documentAddresses_by_ut_temp.clear();
        String html = sciDoc.getHtml();
        String ut = sciDoc.getUt();
        String doc_id = sciDoc.getDocId();
        Map<String, List<Corporation>> doc_addrs_map_corporation_list = new HashMap<>();//此文章的地址对应的单位表
        //获取所有作者缩写名称
        List<String> au_list = ReadHtmlHelper.GetRawValues(html, "AU", "<br[^>]*>");
        if (au_list.size() > authorSizeMax) {
            throw new ListCountTooLongException(au_list.size(), ut);
        }
        //获取doi
        String doi = ReadHtmlHelper.GetRawValue(html, "DI");
        if (!StringUtils.isEmpty(doi)) {
            SciMapUtDoi smut = new SciMapUtDoi(ut, doi);
            sciMapUtDoi_temp.add(smut);
            smut = null;
        }
        //获取所有作者全拼名称
        List<String> af_list = ReadHtmlHelper.GetRawValues(html, "AF", "<br[^>]*>");

        //获取所有作者地址信息(作者名为全称)
        List<String> c1_list = ReadHtmlHelper.GetRawValues(html, "C1", "<br[^>]*>");
        boolean isHasAuthor = false;//是否有作者
        if (ReadHtmlHelper.GetRawValue(html, "C1") != null && ReadHtmlHelper.GetRawValue(html, "C1").indexOf("[") >= 0) {
            isHasAuthor = true;
        }

        //按ut号从document_address表获取相关的地址和单位数据
//        documentAddresses_by_ut_temp = documentAddressService.getListByDocmentId(sciDoc.getUt());
        //获取C1地址信息
        for (int i = 0; i < c1_list.size(); i++) {
            String author = "";
            String addr = "";
            Pattern p = Pattern.compile("(\\[(?<author>.*)\\])?\\s*(?<addr>[^$\\.]+)");
            Matcher m = p.matcher(c1_list.get(i));
            while (m.find()) {
                author = m.group("author");
                addr = m.group("addr");
//                Map<String, String> authorList = new HashMap<>();
                //插入sci_map_au_ad表
                if (author != null && !author.equals("")) {
                    String[] af_arr = author.split(";");
                    for (String af_str : af_arr) {
                        int index_af = af_list.indexOf(CommonUtils.trimAndRegex(af_str));
                        if (index_af >= 0) {
//                            authorList.put(ut + "AU" + (index_af + 1), af_str);
                            SciMapAuAd sciMapAuAd = new SciMapAuAd(ut + "AU" + (index_af + 1), ut + "AD" + (i + 1));
                            sciMapAuAds_temp.add(sciMapAuAd);
                            sciMapAuAd = null;
                        }
                    }
                }
                //插入sci_ad表
                if (addr != null) {
                    SciAd sciAd = new SciAd();
                    sciAd.setAdId(ut + "AD" + (i + 1));
                    sciAd.setAddress(CommonUtils.trimAndRegex(addr));
                    sciAd.setDocId(doc_id);
                    sciAd.setIsReprint(false);
                    sciAd.setIsFirst(i == 0);
                    sciAds_temp.add(sciAd);
                    sciAds_solr.add(sciAd);
//                    InsertSciMapAuadCorpFormAddr(CommonUtils.trimAndRegex(addr), sciAd.getAdId(), sciMapAdCorps_temp);
                    InsertSciMapAdCorporationFormAddr(addr, sciAd.getAdId(), doc_id, sciMapAdCorps_temp, doc_addrs_map_corporation_list);
                    sciAd = null;
                }
                break;
            }
        }
        //获取通讯地址信息(作者名为简称)
        int cl_size = c1_list.size();
        reprintAuthor_temp.clear();
        String rp = ReadHtmlHelper.GetRawValue(html, "RP");
        if (rp != null && rp.length() > 0) {
            String[] rp_arr = rp.split(";");
            Pattern p = Pattern.compile("(?<author>.*)\\(reprint author\\),(?<address>.+)?[\\.$]");
            for (int i = 0; i < rp_arr.length; i++) {
                Matcher m = p.matcher(rp_arr[i]);
                while (m.find()) {
                    String address = m.group("address");
                    String authorStr = m.group("author");
                    //插入sci_ad表
                    if (address != null && !address.equals("")) {
                        SciAd sciAd = new SciAd();
                        sciAd.setAdId(ut + "AD" + (cl_size + i + 1));
                        sciAd.setAddress(CommonUtils.trimAndRegex(address));
                        sciAd.setDocId(doc_id);
                        sciAd.setIsReprint(true);
                        sciAd.setIsFirst(false);
                        sciAds_temp.add(sciAd);
                        sciAds_solr.add(sciAd);
//                        InsertSciMapAuadCorpFormAddr(CommonUtils.trimAndRegex(address), sciAd.getAdId(), sciMapAdCorps_temp);
                        InsertSciMapAdCorporationFormAddr(address, sciAd.getAdId(), doc_id, sciMapAdCorps_temp, doc_addrs_map_corporation_list);
                        sciAd = null;
                    }
                    //添加通讯作者到临时变量中(简称)
                    if (authorStr != null && !authorStr.equals("")) {
                        String[] au_arr = authorStr.split(";");
                        for (String au_str : au_arr) {
                            reprintAuthor_temp.add(CommonUtils.trimAndRegex(au_str));
                            //插入sci_map_au_ad表
                            int index_au = au_list.indexOf(CommonUtils.trimAndRegex(au_str));
                            SciMapAuAd sciMapAuAd = new SciMapAuAd(ut + "AU" + (index_au + 1), ut + "AD" + (cl_size + i + 1));
                            sciMapAuAds_temp.add(sciMapAuAd);
                            sciMapAuAd = null;
                        }
                    }
                    break;
                }
            }
        }

        //插入sci_au表
        for (int i = 0; i < au_list.size(); i++) {
            SciAu sciAu = new SciAu();
            sciAu.setAuId(ut + "AU" + (i + 1));
            sciAu.setDocId(doc_id);
            sciAu.setFullname(CommonUtils.trimAndRegex(af_list.get(i)));
            sciAu.setShortname(CommonUtils.trimAndRegex(au_list.get(i)));
            sciAu.setChinese("");
            sciAu.setIsFirst(i == 0);
            sciAu.setIsReprint(reprintAuthor_temp.indexOf(au_list.get(i)) >= 0);
            String oi = ReadHtmlHelper.GetRawValue2(html, "OI");
            String ri = ReadHtmlHelper.GetRawValue2(html, "RI");
            Pattern p = Pattern.compile("<display_name>(?<name>[^<]+)</display_name>((?!</td><td>).)*</td><td>(?<id>((?!</td>).)*)</td>");
            if (oi != null) {
                Matcher m_oi = p.matcher(oi);
                while (m_oi.find()) {
                    String m_id = m_oi.group("id");
                    String name = m_oi.group("name");
                    if (name != null) {
                        String oi_str = m_id.trim().replaceAll("<[^>]*>", "").replaceAll("[^-\\da-zA-Z]", "");
                        int index_af = af_list.indexOf(CommonUtils.trimAndRegex(name));
                        if (index_af == i) {
                            sciAu.setOi(oi_str);
                        }
                    }
                }
            }
            if (ri != null) {
                Matcher m_ri = p.matcher(ri);
                while (m_ri.find()) {
                    String m_id = m_ri.group("id");
                    String name = m_ri.group("name");
                    if (name != null) {
                        String ri_str = m_id.trim().replaceAll("<[^>]*>", "").replaceAll("[^-\\da-zA-Z]", "");
                        int index_af = af_list.indexOf(CommonUtils.trimAndRegex(name));
                        if (index_af == i) {
                            sciAu.setRi(ri_str);
                        }
                    }
                }
            }

            sciAus_temp.add(sciAu);
            sciAus_solr.add(sciAu);
            sciAu = null;
        }

        //处理solr相关数据
        if (needInsertSolr) {
            SciDocSolrModel sciDocSolrModel = handleSciDocSolr(sciDoc, doc_addrs_map_corporation_list);
            sciDocSolrModelList.add(sciDocSolrModel);
            sciDocSolrModel = null;
        }
    }


    /**
     * 根据地址从corporation_term表的缓存中找出对应的corp_id，插入到map表中
     *
     * @param addrStr 作者的联系地址字符串
     * @param ad_id   作者的联系地址ID
     */
    private void InsertSciMapAdCorporationFormAddr(String addrStr, String ad_id, String doc_id, List<SciMapAdCorporation> sciMapAdCorpsIns, Map<String, List<Corporation>> doc_addrs_map_corporation_list) {
        //将地址按逗号拆分
//        String addr_trim = CommonUtils.trimAndRegex2(addrStr);
        //将地址以逗号分割
        /*String[] addr_arr = addrStr.split(",");
        List<String> termBaseList = new ArrayList<>();
        for (String addr_term : addr_arr) {
            //trim,放入list
            termBaseList.add(addr_term.trim());
        }*/
        //从数据库中模糊搜索出相关的单位数据
        /*List<CorporationTerm> corporationTerms = corporationTermService.getAllByTermBase(termBaseList);
        if (corporationTerms != null) {
            for (CorporationTerm corporationTerm : corporationTerms) {
                String baseAnd = CommonUtils.trimAndRegex2(corporationTerm.getTermAnd());
                String baseNot = CommonUtils.trimAndRegex2(corporationTerm.getTermNot());
                boolean and = true;
                boolean not = false;
                if (!StringUtils.isEmpty(baseAnd)) {
                    and = addr_trim.matches(".*" + baseAnd + ".*");
                }
                if (!StringUtils.isEmpty(baseNot)) {
                    not = addr_trim.matches(".*" + baseNot + ".*");
                }
                if (and && !not) {
                    Integer corpId = corporationTerm.getCorpId();
                    SciMapAdCorporation sciMapAdCorporation = new SciMapAdCorporation(ad_id, corpId);
                    sciMapAdCorpsIns.add(sciMapAdCorporation);
                    if (corporationMap.containsKey(corpId)) {
                        String addr = CommonUtils.trimAndRegex(addrStr);
                        if (doc_addrs_map_corporation_list.containsKey(addr)) {
                            doc_addrs_map_corporation_list.get(addr).add(corporationMap.get(corpId));
                        } else {
                            List<Corporation> corporations = new ArrayList<>();
                            corporations.add(corporationMap.get(corpId));
                            doc_addrs_map_corporation_list.put(addr, corporations);
                        }
                    }
                }
            }
        }*/
        Set<Integer> corp_set = new HashSet<>();
        Map<Integer, Integer> deptIdMapPid = new HashMap<>();
        Set<CorporationDept> dept_set = new HashSet<>();
        Set<Integer> hasPidDeptIdSet = new HashSet<>();
        // 根据地址提取单位
        String addrStr_trim = CommonUtils.trimAndRegex2(addrStr);
        String[] addr_arr = addrStr_trim.split(",");
        for (String addr_item : addr_arr) {
            String addr_trim = addr_item.trim();
            if (corporationTermMap.containsKey(addr_trim)) {
                List<CorporationTerm> corporationTermList = corporationTermMap.get(addr_trim);
                for (CorporationTerm corporationTerm : corporationTermList) {
                    String baseAnd = corporationTerm.getTermAnd();
                    String baseNot = corporationTerm.getTermNot();
                    boolean and = true;
                    boolean not = false;
                    if (!StringUtils.isEmpty(baseAnd)) {
                        and = addrStr_trim.matches(baseAnd);
                    }
                    if (!StringUtils.isEmpty(baseNot)) {
                        not = addrStr_trim.matches(baseNot);
                    }
                    if (and && !not) {
                        Integer corpId = corporationTerm.getCorpId();
                        SciMapAdCorporation sciMapAdCorporation = new SciMapAdCorporation(ad_id, corpId);
                        sciMapAdCorpsIns.add(sciMapAdCorporation);
                        corp_set.add(corpId);
                        if (corporationMap.containsKey(corpId)) {
                            String addr = CommonUtils.trimAndRegex(addrStr);
                            if (doc_addrs_map_corporation_list.containsKey(addr)) {
                                doc_addrs_map_corporation_list.get(addr).add(corporationMap.get(corpId));
                            } else {
                                List<Corporation> corporations = new ArrayList<>();
                                corporations.add(corporationMap.get(corpId));
                                doc_addrs_map_corporation_list.put(addr, corporations);
                            }
                        }
                    }
                }
            }
        }
        for (Integer corpId : corp_set) {
            if (corporationDeptTermMap.containsKey(corpId)) {
                Map<String, List<CorporationDept>> dept_term_map = corporationDeptTermMap.get(corpId);
                for (String addr_item : addr_arr) {
                    String addr_trim = addr_item.trim();
                    if (dept_term_map.containsKey(addr_trim)) {
                        List<CorporationDept> deptList = dept_term_map.get(addr_trim);
                        dept_set.addAll(deptList);
                        for (CorporationDept dept : deptList) {
                            deptIdMapPid.put(dept.getId(), dept.getPid());
                            if (dept.getPid() > 0) {
                                hasPidDeptIdSet.add(dept.getId());
                            }
                        }

                    }
                }
            }
        }
        //根据addr来提取相关的部门
//        if (ad_id.equalsIgnoreCase("000251965000007AD3")) {
//            System.out.println(1);
//        }
        //如果有pid大于0的部门，则此地址只处理此部门
        if (hasPidDeptIdSet.size() > 0) {
            for (Integer deptId : hasPidDeptIdSet) {
                //如果deptId包含pid，也就是此地址中的一级部门为二级部门的父级
                if (deptIdMapPid.containsKey(deptIdMapPid.get(deptId)) || deptIdMapPid.size() == 1) {
                    SciMapDeptDoc sciMapDeptDoc = new SciMapDeptDoc(deptId, doc_id);
                    sciMapdeptDoc_temp.add(sciMapDeptDoc);
                    SciMapDeptAd sciMapDeptAd = new SciMapDeptAd(deptId, ad_id);
                    sciMapDeptAd_temp.add(sciMapDeptAd);
                }
            }
        } else {
            //如果没有pid大于0的部门，则处理所有的部门
            for (Integer deptId : deptIdMapPid.keySet()) {
                SciMapDeptDoc sciMapDeptDoc = new SciMapDeptDoc(deptId, doc_id);
                sciMapdeptDoc_temp.add(sciMapDeptDoc);
                SciMapDeptAd sciMapDeptAd = new SciMapDeptAd(deptId, ad_id);
                sciMapDeptAd_temp.add(sciMapDeptAd);
            }
        }
    }


    /*  @Transactional()
      private void commitSciAuad() {
          sciAuadRepository.save(sciAuads_temp);
      }

      @Transactional()
      private void commitSciMapAuadCorps() {
          sciMapAuadCorpRepository.save(sciMapAuadCorps_temp);
      }

      @Transactional()
      private void commitSciMapAuadCorps() {
          sciMapAuadCorpRepository.save(sciMapAuadCorps_temp);
      }

      @Transactional()
      private void commitSciMapUtDoi() {
          sciMapUtDoiRepository.save(sciMapUtDoi_temp);
      }
  */
    @Transactional()
    void commitSciMapAuAd() {
        sciMapAuadRepository.save(sciMapAuAds_temp);
    }

    @Transactional()
    void commitSciAu() {
        sciAuRepository.save(sciAus_temp);
    }

    @Transactional()
    void commitSciAd() {
        sciAdRepository.save(sciAds_temp);
    }

    @Transactional()
    void commitSciMapAdCorp() {
        sciMapAdCorporationRepository.save(sciMapAdCorps_temp);
    }

    @Transactional()
    void commitSciMapDeptAd() {
        sciMapDeptAdRepository.save(sciMapDeptAd_temp);
    }

    @Transactional()
    void commitSciMapDeptDoc() {
        sciMapDeptDocRepository.save(sciMapdeptDoc_temp);
    }

    /*Set<String> corpition = new HashSet<>();
    Set<String> corpition_cn = new HashSet<>();
    Set<String> corptionFirst = new HashSet<>();
    Set<String> corpReprint = new HashSet<>();
    Set<String> corpDouble = new HashSet<>();*/
    Set<String> author = new HashSet<>();
    Set<String> authorFirst = new HashSet<>();
    Set<String> authorReprint = new HashSet<>();
    Set<String> authorDouble = new HashSet<>();
    Set<String> address = new HashSet<>();
    /*    Set<String> country = new HashSet<>();
        Set<String> province = new HashSet<>();
        Set<String> city = new HashSet<>();*/
    List<String> issnList = new ArrayList<>();
    List<String> groupAuhtorList = new ArrayList<>();
    List<String> list_temp = new ArrayList<>();

    Set<String> corp = new HashSet<>();
    Set<String> corpHosp = new HashSet<>();
    Set<String> corpHospBelong = new HashSet<>();
    Set<String> corpFirst = new HashSet<>();
    Set<String> corpRep = new HashSet<>();
    Set<String> corpDouble = new HashSet<>();
    Set<String> corpCity = new HashSet<>();
    Set<String> corpProvince = new HashSet<>();
    Set<String> corpCountry = new HashSet<>();
    Set<String> corpChina = new HashSet<>();
    Set<String> corpChinaFirst = new HashSet<>();
    Set<String> corpChinaRep = new HashSet<>();
    Set<String> corpForeign = new HashSet<>();

    SciDocSolrModel sciDocSolrModel;

    /**
     * 处理scidoc数据中的html，提取相关solr字段
     *
     * @param sciDoc
     * @param doc_addrs_map_corporation_list 文章的地址对应所属的单位信息
     * @return
     */
    private SciDocSolrModel handleSciDocSolr(SciDoc sciDoc, Map<String, List<Corporation>> doc_addrs_map_corporation_list) {
        sciDocSolrModel = new SciDocSolrModel();
        String html = sciDoc.getHtml();
        sciDocSolrModel.setId(sciDoc.getDocId());
        sciDocSolrModel.setTitle(ReadHtmlHelper.GetRawValue(html, "TI"));
        sciDocSolrModel.setDocAbstract(ReadHtmlHelper.GetRawValue(html, "AB"));
        sciDocSolrModel.setBookSeries(ReadHtmlHelper.GetRawValue(html, "SE"));
        String py = ReadHtmlHelper.GetRawValue(html, "PY");
        if (py != null) {
            sciDocSolrModel.setPublicationYear(Integer.parseInt(py));
            String pd = ReadHtmlHelper.GetRawValue(html, "PD");
            if (pd != null) {
                String publicationDate_str = py + " " + pd;
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd", Locale.UK);
                try {
                    sciDocSolrModel.setPublicationDate(sdf.parse(publicationDate_str));
                } catch (ParseException e) {
                }
            }
        }
        sciDocSolrModel.setPublicationName(ReadHtmlHelper.GetRawValue(html, "SO"));
        sciDocSolrModel.setPublisher(ReadHtmlHelper.GetRawValue(html, "PU") + " " + ReadHtmlHelper.GetRawValue(html, "PI") + " " + ReadHtmlHelper.GetRawValue(html, "PA"));
        sciDocSolrModel.setVolume(ReadHtmlHelper.GetRawValue(html, "VL"));
        sciDocSolrModel.setIssue(ReadHtmlHelper.GetRawValue(html, "IS"));
        sciDocSolrModel.setPage(ReadHtmlHelper.GetRawValue(html, "PG"));
        sciDocSolrModel.setSupplement(ReadHtmlHelper.GetRawValue(html, "SU"));
        sciDocSolrModel.setConferenceName(ReadHtmlHelper.GetRawValue(html, "CT"));
        sciDocSolrModel.setConferenceLocation(ReadHtmlHelper.GetRawValue(html, "CL"));
        sciDocSolrModel.setConferenceDate(ReadHtmlHelper.GetRawValue(html, "CY"));
        sciDocSolrModel.setPissn(ReadHtmlHelper.GetRawValue(html, "SN"));
        sciDocSolrModel.setEissn(ReadHtmlHelper.GetRawValue(html, "EI"));
        issnList.clear();
        if (!StringUtils.isEmpty(sciDocSolrModel.getPissn())) {
            issnList.add(sciDocSolrModel.getPissn());
        }
        if (!StringUtils.isEmpty(sciDocSolrModel.getEissn())) {
            issnList.add(sciDocSolrModel.getEissn());
        }
        if (issnList.size() > 0) {
            sciDocSolrModel.setIssn(new ArrayList<>(issnList));
        }
        sciDocSolrModel.setIsbn(ReadHtmlHelper.GetRawValue(html, "BN"));
        groupAuhtorList.clear();
        if (ReadHtmlHelper.GetRawValue(html, "GP") != null) {
            String[] groupAuhtorArr = ReadHtmlHelper.GetRawValue(html, "GP").split(";");
            for (String g : groupAuhtorArr) {
                groupAuhtorList.add(g.trim());
            }
            sciDocSolrModel.setGroupAuhtor(new ArrayList<>(groupAuhtorList));
        }
        //机构，转大写
        ///机构（第一）
        //机构（通讯）
        //机构（第一或通讯）
        //作者，转大写
        //作者（第一）
        //作者（通讯）
        //作者（第一或通讯）
        //地址
        //国家
        //省
        //市
//        corpition.clear();
//        corpition_cn.clear();
//        corptionFirst.clear();
//        corpReprint.clear();
//        corpDouble.clear();
        author.clear();
        authorFirst.clear();
        authorReprint.clear();
        authorDouble.clear();
        address.clear();
//        country.clear();
//        province.clear();
//        city.clear();
        corp.clear();
        corpHosp.clear();
        corpHospBelong.clear();
        corpFirst.clear();
        corpRep.clear();
        corpDouble.clear();
        corpCity.clear();
        corpProvince.clear();
        corpCountry.clear();
        corpChina.clear();
        corpChinaFirst.clear();
        corpChinaRep.clear();
        corpForeign.clear();
        if (sciAds_solr != null) {
            for (com.zxkj.domain.entity.SciAd sciAd : sciAds_solr) {
                address.add(sciAd.getAddress());
//                sciMapAdCorps_solr.clear();
//                InsertSciMapAuadCorpFormAddr(sciAd.getAddress(), sciAd.getAdId(), sciMapAdCorps_solr);
                if (doc_addrs_map_corporation_list.containsKey(sciAd.getAddress())) {
                    for (Corporation corporation : doc_addrs_map_corporation_list.get(sciAd.getAddress())) {
                        //处理corp相关
                        String corpName = corporation.getNewname();
                        corp.add(corpName);
                        if (corpName.contains("医院")) {
                            corpHosp.add(corpName);
                            corpHospBelong.add(corpName);
                            //如果此医院有上级单位，则一并插入belong中,并将上级单位插入corp中
                            if (corporation.getPid() != null && corporation.getPid() > 0 && corporationMap.containsKey(corporation.getPid())) {
                                corpHospBelong.add(corporationMap.get(corporation.getPid()).getNewname());
                            }
                        }
                        String city = corporation.getCity();
                        if (!StringUtils.isEmpty(city)) {
                            corpCity.add(city);
                        }
                        String country = corporation.getCountry();
                        if (!StringUtils.isEmpty(country)) {
                            corpCountry.add(country);
                        }
                        String province = corporation.getProvince();
                        if (!StringUtils.isEmpty(province)) {
                            corpProvince.add(province);
                        }
                        if (sciAd.getIsFirst()) {
                            corpFirst.add(corpName);
                        }
                        if (sciAd.getIsReprint()) {
                            corpRep.add(corpName);
                        }
                        if (sciAd.getIsFirst() && sciAd.getIsReprint()) {
                            corpDouble.add(corpName);
                        }
                        if (country.equalsIgnoreCase("PEOPLES R CHINA")) {
                            corpChina.add(corpName);
                            if (sciAd.getIsFirst()) {
                                corpChinaFirst.add(corpName);
                            }
                            if (sciAd.getIsReprint()) {
                                corpChinaRep.add(corpName);
                            }
                        } else {
                            corpForeign.add(corpName);
                        }
                    }
                }
            }
        }
        if (sciAus_solr != null) {
            for (com.zxkj.domain.entity.SciAu sciAu : sciAus_solr) {
                author.add(sciAu.getFullname());
                if (sciAu.getIsFirst()) {
                    authorFirst.add(sciAu.getFullname());
                }
                if (sciAu.getIsReprint()) {
                    authorReprint.add(sciAu.getFullname());
                }
                if (sciAu.getIsFirst() || sciAu.getIsReprint()) {
                    authorDouble.add(sciAu.getFullname());
                }
            }
        }

        /*if (corpition.size() > 0) {
            sciDocSolrModel.setCorpition(new ArrayList<>(corpition));
        }
        if (corpition_cn.size() > 0) {
            sciDocSolrModel.setCorpitionCn(new ArrayList<>(corpition_cn));
        }
        if (corptionFirst.size() > 0) {
            sciDocSolrModel.setCorptionFirst(new ArrayList<>(corptionFirst));
        }
        if (corpReprint.size() > 0) {
            sciDocSolrModel.setCorpReprint(new ArrayList<>(corpReprint));
        }
        if (corpDouble.size() > 0) {
            sciDocSolrModel.setCorpDouble(new ArrayList<>(corpDouble));
        }*/
        if (author.size() > 0) {
            sciDocSolrModel.setAuthor(new ArrayList<>(author));
        }
        if (authorFirst.size() > 0) {
            sciDocSolrModel.setAuthorFirst(new ArrayList<>(authorFirst));
        }
        if (authorReprint.size() > 0) {
            sciDocSolrModel.setAuthorReprint(new ArrayList<>(authorReprint));
        }
        if (authorDouble.size() > 0) {
            sciDocSolrModel.setAuthorDouble(new ArrayList<>(authorDouble));
        }
        if (address.size() > 0) {
            sciDocSolrModel.setAddress(new ArrayList<>(address));
        }
        if (corp.size() > 0) {
            sciDocSolrModel.setCorp(new ArrayList<>(corp));
        }
        if (corpHosp.size() > 0) {
            sciDocSolrModel.setCorpHosp(new ArrayList<>(corpHosp));
        }
        if (corpHospBelong.size() > 0) {
            sciDocSolrModel.setCorpHospBelong(new ArrayList<>(corpHospBelong));
        }
        if (corpFirst.size() > 0) {
            sciDocSolrModel.setCorpFirst(new ArrayList<>(corpFirst));
        }
        if (corpRep.size() > 0) {
            sciDocSolrModel.setCorpRep(new ArrayList<>(corpRep));
        }
        if (corpDouble.size() > 0) {
            sciDocSolrModel.setCorpDouble(new ArrayList<>(corpDouble));
        }
        if (corpCity.size() > 0) {
            sciDocSolrModel.setCorpCity(new ArrayList<>(corpCity));
        }
        if (corpProvince.size() > 0) {
            sciDocSolrModel.setCorpProvince(new ArrayList<>(corpProvince));
        }
        if (corpCountry.size() > 0) {
            sciDocSolrModel.setCorpCountry(new ArrayList<>(corpCountry));
        }
        if (corpChina.size() > 0) {
            sciDocSolrModel.setCorpChina(new ArrayList<>(corpChina));
        }
        if (corpChinaFirst.size() > 0) {
            sciDocSolrModel.setCorpChinaFirst(new ArrayList<>(corpChinaFirst));
        }
        if (corpChinaRep.size() > 0) {
            sciDocSolrModel.setCorpChinaRep(new ArrayList<>(corpChinaRep));
        }
        if (corpForeign.size() > 0) {
            sciDocSolrModel.setCorpForeign(new ArrayList<>(corpForeign));
        }

        /*if (country.size() > 0) {
            sciDocSolrModel.setCountry(new ArrayList<>(country));
        }
        if (province.size() > 0) {
            sciDocSolrModel.setProvince(new ArrayList<>(province));
        }
        if (city.size() > 0) {
            sciDocSolrModel.setCity(new ArrayList<>(city));
        }*/

        //author2author作者和作者的全部組合
        /*if (sciDocSolrModel.getAuthor() != null && sciDocSolrModel.getAuthor().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getAuthor().size(); i++) {
                for (int j = i + 1; j < sciDocSolrModel.getAuthor().size(); j++) {
                    list_temp.add(sciDocSolrModel.getAuthor().get(i) + "+" + sciDocSolrModel.getAuthor().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setAuthor2author(new ArrayList<>(list_temp));
        }*/

        //corporation2corporation单位和单位的全部组合
        /*if (sciDocSolrModel.getCorpition() != null && sciDocSolrModel.getCorpition().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getCorpition().size(); i++) {
                for (int j = i + 1; j < sciDocSolrModel.getCorpition().size(); j++) {
                    list_temp.add(sciDocSolrModel.getCorpition().get(i) + "+" + sciDocSolrModel.getCorpition().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setCorporation2corporation(new ArrayList<>(list_temp));
        }*/


        String documentType_str = ReadHtmlHelper.GetRawValue(html, "DT");
        if (documentType_str != null) {
            sciDocSolrModel.setDocumentType(getListFromSplit(documentType_str, ";", false));
        }

        String authorKeyword_str = ReadHtmlHelper.GetRawValue(html, "DE");
        if (authorKeyword_str != null) {
            sciDocSolrModel.setAuthorKeyword(getListFromSplit(authorKeyword_str, ";", true));
        }

        //corporation2keyword单位和关键字的全部组合
        /*if (sciDocSolrModel.getCorpition() != null && sciDocSolrModel.getCorpition().size() > 0 && sciDocSolrModel.getAuthorKeyword() != null && sciDocSolrModel.getAuthorKeyword().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getCorpition().size(); i++) {
                for (int j = 0; j < sciDocSolrModel.getAuthorKeyword().size(); j++) {
                    list_temp.add(sciDocSolrModel.getCorpition().get(i) + "+" + sciDocSolrModel.getAuthorKeyword().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setCorporation2keyword(new ArrayList<>(list_temp));
        }*/

        //author2keyword作者和关键字的全部组合
        if (sciDocSolrModel.getAuthor() != null && sciDocSolrModel.getAuthor().size() > 0 && sciDocSolrModel.getAuthorKeyword() != null && sciDocSolrModel.getAuthorKeyword().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getAuthor().size(); i++) {
                if (i == 20) {
                    break;
                }
                for (int j = 0; j < sciDocSolrModel.getAuthorKeyword().size(); j++) {
                    list_temp.add(sciDocSolrModel.getAuthor().get(i) + "+" + sciDocSolrModel.getAuthorKeyword().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setAuthor2keyword(new ArrayList<>(list_temp));
        }

        //author2author作者和作者的全部组合
        if (sciDocSolrModel.getAuthor() != null && sciDocSolrModel.getAuthor().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getAuthor().size(); i++) {
                if (i == 20) {
                    break;
                }
                for (int j = 0; j < sciDocSolrModel.getAuthor().size(); j++) {
                    if (j == 20) {
                        break;
                    }
                    if (i == j) {
                        break;
                    }
                    list_temp.add(sciDocSolrModel.getAuthor().get(i) + "+" + sciDocSolrModel.getAuthor().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setAuthor2author(new ArrayList<>(list_temp));
        }

        //keyword2keyword 关键词+关键词的全部组合
        if (sciDocSolrModel.getAuthorKeyword() != null && sciDocSolrModel.getAuthorKeyword().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getAuthorKeyword().size(); i++) {
                for (int j = 0; j < sciDocSolrModel.getAuthorKeyword().size(); j++) {
                    if (i == j) {
                        break;
                    }
                    list_temp.add(sciDocSolrModel.getAuthorKeyword().get(i) + "+" + sciDocSolrModel.getAuthorKeyword().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setKeyword2keyword(new ArrayList<>(list_temp));
        }

        //corp2corp 机构+机构的全部组合
        if (sciDocSolrModel.getCorp() != null && sciDocSolrModel.getCorp().size() > 0) {
            list_temp.clear();
            for (int i = 0; i < sciDocSolrModel.getCorp().size(); i++) {
                if (i == 20) {
                    break;
                }
                for (int j = 0; j < sciDocSolrModel.getCorp().size(); j++) {
                    if (j == 20) {
                        break;
                    }
                    if (i == j) {
                        break;
                    }
                    list_temp.add(sciDocSolrModel.getCorp().get(i) + "+" + sciDocSolrModel.getCorp().get(j));
                    if (list_temp.size() >= listSizeMax) {
                        break;
                    }
                }
                if (list_temp.size() >= listSizeMax) {
                    break;
                }
            }
            sciDocSolrModel.setCorp2corp(new ArrayList<>(list_temp));
        }

        String keywordplus_str = ReadHtmlHelper.GetRawValue(html, "ID");
        if (keywordplus_str != null) {
            sciDocSolrModel.setKeywordplus(getListFromSplit(keywordplus_str, ";", false));
        }

        String email_str = ReadHtmlHelper.GetRawValue(html, "EM");
        if (email_str != null) {
            sciDocSolrModel.setEmail(getListFromSplit(email_str, ";", false));
        }

        sciDocSolrModel.setUt(sciDoc.getUt());
        sciDocSolrModel.setCited(sciDoc.getCitedcnt());
        sciDocSolrModel.setCiting(Integer.parseInt(ReadHtmlHelper.GetRawValue(html, "NR")));
        sciDocSolrModel.setCitedDate(sciDoc.getCitedon());

        String category_str = ReadHtmlHelper.GetRawValue(html, "SC");
        if (category_str != null) {
            sciDocSolrModel.setCategory(getListFromSplit(category_str, ";", false));
        }
        String language_str = ReadHtmlHelper.GetRawValue(html, "LA");
        if (language_str != null) {
            sciDocSolrModel.setLanguage(getListFromSplit(language_str, ";", false));
        }
        String researchArea_str = ReadHtmlHelper.GetRawValue(html, "WC");
        if (researchArea_str != null) {
            sciDocSolrModel.setResearchArea(getListFromSplit(researchArea_str, ";", false));
        }
        String sponsor_str = ReadHtmlHelper.GetRawValue(html, "SP");
        if (sponsor_str != null) {
            sciDocSolrModel.setSponsor(getListFromSplit(sponsor_str, ";", false));
        }
        String fundingAgency_str = ReadHtmlHelper.GetRawValue(html, "FO");
        if (fundingAgency_str != null) {
            sciDocSolrModel.setFundingAgency(getListFromSplit(fundingAgency_str, ";", false));
        }
        String grandNumber_str = ReadHtmlHelper.GetRawValue(html, "FG");
        if (grandNumber_str != null) {
            sciDocSolrModel.setGrandNumber(getListFromSplit(grandNumber_str, ";", false));
        }
        //auid获取ORCID和researchID
        list_temp.clear();
        String oi = ReadHtmlHelper.GetRawValue2(html, "OI");
        String ri = ReadHtmlHelper.GetRawValue2(html, "RI");
        Pattern p = Pattern.compile("<display_name>(?<name>[^<]+)</display_name>((?!</td><td>).)*</td><td>(?<id>((?!</td>).)*)</td>");
        if (oi != null) {
            Matcher m_oi = p.matcher(oi);
            while (m_oi.find()) {
                String m_id = m_oi.group("id");
                String oi_str = m_id.trim().replaceAll("<[^>]*>", "").replaceAll("[^-\\da-zA-Z]", "");
                list_temp.add(oi_str);
            }
        }
        if (ri != null) {
            Matcher m_ri = p.matcher(ri);
            while (m_ri.find()) {
                String m_id = m_ri.group("id");
                list_temp.add(m_id.trim().replaceAll("<[^>]*>", "").replaceAll("[^-\\da-zA-Z]", ""));
            }
        }
        if (list_temp.size() > 0) {
            sciDocSolrModel.setAuid(new ArrayList<>(list_temp));
        }

        sciDocSolrModel.setDoi(ReadHtmlHelper.GetRawValue(html, "DI"));
        sciDocSolrModel.setPmid(ReadHtmlHelper.GetRawValue(html, "PM"));

        //影响因子(未找到值为-1)
        //issn取PISSN或EISSN，哪个存在取哪个
        //ESCISUBJECT、SCIJOURAREA、ACSJOURAREA
        String issn = "";
        sciDocSolrModel.setImpactFactor(-1);
        sciDocSolrModel.setIfYear(sciDoc.getPubyear());
        if (!StringUtils.isEmpty(sciDocSolrModel.getPissn())) {
            issn = sciDocSolrModel.getPissn();
            PublicationImpactfactor publicationImpactfactor = getImpactFactor(issn, sciDoc.getPubyear());
            if (publicationImpactfactor != null) {
                sciDocSolrModel.setImpactFactor(((float) (publicationImpactfactor.getImpactFactor())));
                sciDocSolrModel.setIfYear(publicationImpactfactor.getYear());
                if (!issn.equals("")) {
                    if (publication_subjectarea_temp.containsKey(issn + "ESCISubject")) {
                        sciDocSolrModel.setSubjectEsci(publication_subjectarea_temp.get(issn + "ESCISubject"));
                    }
                    if (publication_subjectarea_temp.containsKey(issn + "SCIJournalArea")) {
                        sciDocSolrModel.setJournalAreaSci(publication_subjectarea_temp.get(issn + "SCIJournalArea"));
                    }
                    if (publication_subjectarea_temp.containsKey(issn + "ACSJournalArea")) {
                        sciDocSolrModel.setJournalAreaAcs(publication_subjectarea_temp.get(issn + "ACSJournalArea"));
                    }
                }
            }
        }
        if (StringUtils.isEmpty(sciDocSolrModel.getPissn()) && !StringUtils.isEmpty(sciDocSolrModel.getEissn())) {
            issn = sciDocSolrModel.getEissn();
            PublicationImpactfactor publicationImpactfactor = getImpactFactor(issn, sciDoc.getPubyear());
            if (publicationImpactfactor != null) {
                sciDocSolrModel.setImpactFactor(((float) (publicationImpactfactor.getImpactFactor())));
                sciDocSolrModel.setIfYear(publicationImpactfactor.getYear());
                if (!issn.equals("")) {
                    if (publication_subjectarea_temp.containsKey(issn + "ESCISubject")) {
                        sciDocSolrModel.setSubjectEsci(publication_subjectarea_temp.get(issn + "ESCISubject"));
                    }
                    if (publication_subjectarea_temp.containsKey(issn + "SCIJournalArea")) {
                        sciDocSolrModel.setJournalAreaSci(publication_subjectarea_temp.get(issn + "SCIJournalArea"));
                    }
                    if (publication_subjectarea_temp.containsKey(issn + "ACSJournalArea")) {
                        sciDocSolrModel.setJournalAreaAcs(publication_subjectarea_temp.get(issn + "ACSJournalArea"));
                    }
                }
            }
        }

        //所在数据库
        list_temp.clear();
        list_temp.add("SCI");
        if (!StringUtils.isEmpty(sciDocSolrModel.getPmid())) {
            list_temp.add("PUBMED");
        }
        sciDocSolrModel.setInclude(new ArrayList<>(list_temp));

        return sciDocSolrModel;
    }

    private List<PublicationImpactfactor> publicationImpactfactorList = new ArrayList<>();

    /**
     * 根据issn和年份取出对应的影响因子
     *
     * @param issn
     * @param year
     * @return
     */
    private PublicationImpactfactor getImpactFactor(String issn, Integer year) {
        publicationImpactfactorList.clear();
        for (PublicationImpactfactor publicationImpactfactor : publication_impactfactor_temp) {
            //判断issn是否相等
            if (issn.equals(publicationImpactfactor.getISSN())) {
                publicationImpactfactorList.add(publicationImpactfactor);
                continue;
            }
            if (publicationImpactfactorList.size() > 0) {
                break;
            }
        }
        /*for (int i = 0; i < publicationImpactfactorList.size(); i++) {
            if ((i == 0 && year <= publicationImpactfactorList.get(i).getYear().intValue())
                    || (i == publicationImpactfactorList.size() - 1 && year >= publicationImpactfactorList.get(i).getYear().intValue())
                    || (year == publicationImpactfactorList.get(i).getYear().intValue())) {
                PublicationImpactfactor publicationImpactfactor = new PublicationImpactfactor();
                publicationImpactfactor.setISSN(publicationImpactfactorList.get(i).getISSN());
                publicationImpactfactor.setYear(publicationImpactfactorList.get(i).getYear());
                publicationImpactfactor.setArticleCount(publicationImpactfactorList.get(i).getArticleCount());
                publicationImpactfactor.setTotalCited(publicationImpactfactorList.get(i).getTotalCited());
                publicationImpactfactor.setImpactFactor(publicationImpactfactorList.get(i).getImpactFactor());
                return publicationImpactfactor;
            }
        }*/
        for (int i = 0; i < publicationImpactfactorList.size(); i++) {
            if (year == publicationImpactfactorList.get(i).getYear().intValue()) {
                PublicationImpactfactor publicationImpactfactor = new PublicationImpactfactor();
                publicationImpactfactor.setISSN(publicationImpactfactorList.get(i).getISSN());
                publicationImpactfactor.setYear(publicationImpactfactorList.get(i).getYear());
                publicationImpactfactor.setArticleCount(publicationImpactfactorList.get(i).getArticleCount());
                publicationImpactfactor.setTotalCited(publicationImpactfactorList.get(i).getTotalCited());
                publicationImpactfactor.setImpactFactor(publicationImpactfactorList.get(i).getImpactFactor());
                return publicationImpactfactor;
            }
        }
        return null;
    }


    private List<String> listFormSplitList = new ArrayList<>();

    /**
     * 用分隔符将字符串分隔并生成list
     *
     * @param str   字符串
     * @param split
     * @return
     */
    private List<String> getListFromSplit(String str, String split, boolean isToLower) {
        listFormSplitList.clear();
        if (str != null) {
            String[] arr = str.split(";");
            for (String s : arr) {
                if (isToLower) {
                    listFormSplitList.add(CommonUtils.trimAndRegex2(s.trim()));
                } else {
                    listFormSplitList.add(CommonUtils.trimAndRegex(s.trim()));
                }
            }
        }
        return new ArrayList<>(listFormSplitList);
    }

    /**
     * 仅在单位没有提供作者名称和部门作者对应关系的时候
     * 在插入地址、作者等表后，才可执行此方法
     * 根据单位下的sci_au插入corporation_au，corporation_map_dept_au，sci_map_au_corp_au
     */
    @Override
    public void insertIntoCorpAu() {
        if (corpId > 0) {
            Map<String, String> fullnameMapCorpAuId = new HashMap<>();//fullname对应corpau表的id
            //取出该单位下的所有corp_au
            logger.warn("开始获取该单位下的所有corp_au....");
            List<CorporationAu> corporationAuList = corporationAuRepository.findAllByCorpId(corpId);
            if (corporationAuList != null) {
                logger.warn("该单位下的corp_au条数为：" + corporationAuList.size());
                for (CorporationAu corporationAu : corporationAuList) {
                    fullnameMapCorpAuId.put(CommonUtils.trimAndRegex2(corporationAu.getName()), corporationAu.getId());
                }
            }

            //插入corporation_au
            //获取该单位所有的fullname
            List<SciAu> all_au_list = sciDeptDao.getAuNameByFullnameByCorpId(corpId);
            logger.warn("该单位下的sci_au条数为：" + all_au_list.size());
            for (SciAu sciAu : all_au_list) {
                String corpAuid;
                //判断是否有重名的
                //重名的直接获取对应的corpAuid；没有重名的添加一行新corpau的数据
                String fullname_trim = CommonUtils.trimAndRegex2(sciAu.getFullname());
                if (fullnameMapCorpAuId.containsKey(fullname_trim)) {
                    corpAuid = fullnameMapCorpAuId.get(fullname_trim);
                } else {
                    corpAuid = UUID.randomUUID().toString();
                    CorporationAu corporationAu = new CorporationAu();
                    corporationAu.setId(corpAuid);
                    corporationAu.setCorpId(corpId);
                    corporationAu.setName(sciAu.getFullname());
                    corporationAus_temp.add(corporationAu);
                    fullnameMapCorpAuId.put(fullname_trim, corpAuid);
                }

                SciMapAuCorpAu sciMapAuCorpAu = new SciMapAuCorpAu(sciAu.getAuId(), corpAuid);
                sciMapAuCorpAus_temp.add(sciMapAuCorpAu);
            }

            Map<String, CorporationDept> notDeptAuthor = new HashMap<>();//属于一级部门，但是没有二级部门的作者
            Set<String> hasDeptAuthor = new HashSet<>();//已经有二级部门所属
            //取出该单位下的所有部门
            List<CorporationDept> corporationDeptList = corporationDeptRepository.findAllByCorpId(corpId);
            logger.warn("该单位部门个数：" + corporationDeptList.size());
            for (CorporationDept dept : corporationDeptList) {
//                logger.warn("开始处理部门：" + dept.getDeptName());
                List<SciAu> sciAuList = sciDeptDao.getAuNameByFullnameByDeptId(dept.getId());
                if (sciAuList != null) {
                    for (SciAu sciAu : sciAuList) {
                        String fullname_trim = CommonUtils.trimAndRegex2(sciAu.getFullname());
                        if (fullnameMapCorpAuId.containsKey(fullname_trim)) {
                            if (dept.getPid() == 0 && !notDeptAuthor.containsKey(fullname_trim) && !hasDeptAuthor.contains(fullname_trim)) {
                                notDeptAuthor.put(fullname_trim, dept);
                                continue;
                            } else if (dept.getPid() > 0) {
                                String corpAuid = fullnameMapCorpAuId.get(fullname_trim);
                                CorporationMapDeptAu corporationMapDeptAu = new CorporationMapDeptAu(dept.getId(), corpAuid);
                                corporationMapDeptAu_temp.add(corporationMapDeptAu);
                                if (notDeptAuthor.containsKey(fullname_trim)) {
                                    notDeptAuthor.remove(fullname_trim);
                                    hasDeptAuthor.add(fullname_trim);
                                }
                            }
                        }

                    }
                }
            }
            //将属于一级部门但是没有二级部门的入库
            for (String fullname : notDeptAuthor.keySet()) {
                String corpAuid = fullnameMapCorpAuId.get(fullname);
                CorporationMapDeptAu corporationMapDeptAu = new CorporationMapDeptAu(notDeptAuthor.get(fullname).getId(), corpAuid);
                corporationMapDeptAu_temp.add(corporationMapDeptAu);
            }

            logger.warn("插入corporation_au表：(" + corporationAus_temp.size() + "条)");
            if (corporationAus_temp.size() > 0) {
                corporationAuRepository.save(corporationAus_temp);
            }
            logger.warn("插入corporation_map_dept_au表：(" + corporationMapDeptAu_temp.size() + "条)");
            if (corporationMapDeptAu_temp.size() > 0) {
                corporationMapDeptAuRepository.save(corporationMapDeptAu_temp);
            }
            logger.warn("插入sci_map_au_corp_au表：(" + sciMapAuCorpAus_temp.size() + "条)");
            if (sciMapAuCorpAus_temp.size() > 0) {
                sciMapAuCorpAuRepository.save(sciMapAuCorpAus_temp);
            }

        }
    }

}
